package 地区数据_分省年度数据;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import entity.Table;
import okhttp3.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;

import java.io.*;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URLEncoder;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class 社会消费品零售总额 {
    //ip代理
    static String zhimaLink = "http://tunnel-api.apeyun.com/q?id=2120032600016801539&secret=ASySTfjht9VuRUh9&limit=1&format=txt&auth_mode=auto";
    public static Map<String, String> provance_map = null;
    static String[] yearsArray = new String[]{"2016","2017", "2018"};
    //    static String[] yearsArray = new String[]{"1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015", "2016", "2017", "2018"};
    public static void main(String[] args) throws IOException {
        getData();
    }

    private static void getData() throws IOException {
        Map<String, String> table_id_name = getMap();
        //获取地区id与名称键值对
        provance_map = ProvanceJsonString.getProvance_map();//获取省份编码与名称键值对
        String url = "http://data.stats.gov.cn/easyquery.htm?cn=E0103";
        List<String> keyList = getKeySet(table_id_name);
        for (int n = 0; n < keyList.size(); n++) {
            //先获取第一层中的所有的key
            String areaId = keyList.get(n);//第一层id
            if (!areaId.contains("A0H")) {
                continue;
            }
            String rootName = table_id_name.get(areaId);//第一层id名称
            List<String> provanceCodeList = getKeySet(provance_map);
            int rowNumber = 0;
            String tableId = areaId;
            String tableName = rootName;
            File file = new File("E:\\程存淦_处理程序2020\\王佳艺\\CRED\\国家数据\\地区数据_分省年度数据\\" + tableName);
            if (!file.exists()) {
                file.mkdirs();//如果不存在就新建
            }
            //开始导出数据
            exportData(provanceCodeList, rowNumber, tableId, tableName);
        }

    }

    /**
     * 使用猿人云代理更换ip
     *
     * @param zhimaLink
     * @return
     */
    private static String getIP(String zhimaLink) throws IOException {
        //获取IP的程序
        List<String> Info_List = new ArrayList<String>();
        com.squareup.okhttp.Request request = new com.squareup.okhttp.Request.Builder().url(zhimaLink).build();
        com.squareup.okhttp.OkHttpClient client2 = new com.squareup.okhttp.OkHttpClient();
        com.squareup.okhttp.Response response = client2.newCall(request).execute();
        String html = new String(response.body().bytes(), "UTF-8");
        return html;
    }

    /**
     * \s是指空白，包括空格、换行、tab缩进等所有的空白，而\S刚好相反，[\s\S]*是完全通配的意思
     * (.+)默认是贪婪匹配
     * 从eval( 到{}))的正则表达式
     */
    static Pattern compile = Pattern.compile("[\\s\\S]*(eval\\(.+[\\s\\S]*\\{\\}\\)\\))[\\s\\S]*");
    private static String requestWZWS(String html, String wzws_cid, OkHttpClient okHttpClient) throws IOException {
        Elements select = Jsoup.parse(html).select("[type=\"text/javascript\"]");
        String js = select.html();
        Matcher matcher = compile.matcher(js);
        String group = "";
        if (matcher.matches()) {
            group = matcher.group(1);
        }
        String s = invokeJsFromNode(group);
        String wzws_cid1 = null;
        while (true){
            String url1 = "http://data.stats.gov.cn" + s;
            System.out.println(url1);
            Request request1 = new Request.Builder().url(url1)
                    .header("Cookie", wzws_cid)
                    .header("Referer", "http://data.stats.gov.cn/easyquery.htm?cn=E0103")
                    .header("Accept-Language", "zh-CN,zh;q=0.9")
                    .header("Connection", "keep-alive")
                    .header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36")
                    .get().build();
            Call call = okHttpClient.newCall(request1);
            Response execute = call.execute();
            String string = execute.body().string();
            System.out.println(string);
            wzws_cid1  = execute.header("Set-Cookie").split("path=/")[0];
            return wzws_cid1;
        }
    }

    public static String requestIndex(OkHttpClient client){
        String url1 = "http://data.stats.gov.cn";
        System.out.println(url1);
        Request request1 = new Request.Builder().url(url1)
                .header("Cookie", "")
                .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3")
                .header("Accept-Language", "zh-CN,zh;q=0.9")
                .header("Connection", "keep-alive")
                .header("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")
                .get().build();
        Call call = client.newCall(request1);
        Response execute = null;
        try {
            execute = call.execute();
            String string = execute.body().string();
            if(string.contains("请开启JavaScript并刷新该页") || string.contains("307 Temporary Redirect")){
                //先获取wzws_cid的值
                String wzws_cid = execute.header("Set-Cookie").split("path=/")[0];
                wzws_cid = requestWZWS(string, wzws_cid,client);
                return wzws_cid;
            }
            System.out.println(string);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return "";
    }
    private static String invokeJsFromNode(String js) throws IOException {
        String s = IOUtils.toString(new FileInputStream("a.js"), "utf-8");
        String pathname = UUID.randomUUID() + ".js";//返回一个随机生成的UUID
        File file = new File(pathname);
        FileUtils.write(file,js+s);
        ProcessBuilder builder = new ProcessBuilder(
                "cmd.exe", "/c", "node "+pathname);
        builder.redirectErrorStream(true);
        Process p = null;
        try {
            p = builder.start();
        } catch (IOException e) {
            e.printStackTrace();
        }
        System.out.println(p);
//        String result = IOUtils.toString(p.getInputStream(), "utf-8");
        BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
        List<String> strings = null;
        try {
            strings = IOUtils.readLines(r);
        } catch (Exception e) {
            e.printStackTrace();
        }
        file.delete();
        return strings.get(1);
    }
    public static void exportData(List<String> provanceCodeList, int rowNumber, String tableKeyId, String tableName) {
        XSSFWorkbook workbook = new XSSFWorkbook();
        XSSFSheet sheet = workbook.createSheet();
        for (int y = 0; y < yearsArray.length; y++) {
            String year = null;
            year = yearsArray[y];
            List<String> momeList = new ArrayList<>();
            for (String code : provanceCodeList) {
                String contentUrl = "http://data.stats.gov.cn/easyquery.htm?m=QueryData&dbcode=fsnd&rowcode=zb&colcode=sj";
                rowNumber++;
                String wdsStr = "[{\"wdcode\":\"reg\",\"valuecode\":\"" + code + "\"}]";
                String dfwdsStr = "[{\"wdcode\":\"zb\",\"valuecode\":\"" + tableKeyId + "\"},{\"wdcode\":\"sj\",\"valuecode\":\"" + year + "\"}]";
                List<String> oneProvanceValueList = new ArrayList<String>();
                String area = provance_map.get(code);
                oneProvanceValueList.add(area);
                oneProvanceValueList.add(year+"年");
                try {
                    String encode_wds = URLEncoder.encode(wdsStr, "UTF-8");//utf-8转码为endode
                    String encode_dfwds = URLEncoder.encode(dfwdsStr, "UTF-8");
                    contentUrl = contentUrl + "&wds=" + encode_wds + "&dfwds=" + encode_dfwds;
                    String jsonString ="";
                    int chongshi=0;
                    while (true) {
                        jsonString = getContentJsonStr(contentUrl);
                        if (jsonString.contains("访问验证") || jsonString.contains("请开启JavaScript并刷新该页")|| jsonString.equals("")) {
                            chongshi++;
                            //如果再再短短的10秒内重试了10次以上就休眠20秒继续(因为ip在某个时间段不是很好)
                            if (chongshi>10){
                                try {
                                    System.out.println("休眠3分钟。。。。。。");
                                    Thread.sleep(180000);
                                } catch (InterruptedException e) {
                                    e.printStackTrace();
                                }
                            }
                            System.out.println("重试");
                            String IP_Info;
                            try {
                                IP_Info = getIP(zhimaLink);
                            }catch (Exception e1){
                                continue;
                            }
                            String ip = null;
                            String host = null;
                            try {
                                ip = IP_Info.split(":")[0];
                                host = IP_Info.split(":")[1];
                            } catch (Exception e) {
                                e.printStackTrace();
                                continue;
                            }
                            System.getProperties().setProperty("http.proxyHost", ip);
                            System.getProperties().setProperty("http.proxyPort", host);
                            continue;
                        }
                        System.out.println(jsonString);
                        break;
                    }

                    JSONObject jsonObj = JSON.parseObject(jsonString);//会出现访问验证，如果出现访问验证码验证，重试加换ip
                    JSONObject returndataObj = jsonObj.getJSONObject("returndata");
                    JSONArray wdnodesArray = returndataObj.getJSONArray("wdnodes");
                    JSONObject titleObj = wdnodesArray.getJSONObject(0);
                    JSONArray nodesArray = titleObj.getJSONArray("nodes");
                    String[] chineseNames = new String[nodesArray.size() + 2];
                    chineseNames[0] = "地区";
                    chineseNames[1] = "年份";
                    for (int i = 0; i < nodesArray.size(); i++) {
                        JSONObject oneTitleObj = nodesArray.getJSONObject(i);
                        String cname = oneTitleObj.getString("cname");
                        String unit = oneTitleObj.getString("unit");
                        String memo = oneTitleObj.getString("memo");
                        if (memo.length() > 3) {
                            if (!momeList.contains(memo)) {
                                momeList.add(memo);
                            }
                        }
                        String tableTitle = cname + "/" + unit;
                        if (tableTitle.contains("(上年=100)")) {
                            tableTitle.replace("(上年=100)/","(上年=100)/");
                        }
                        if (tableTitle.contains("帐款")) {
                            tableTitle.replace("帐款","账款");
                        }
                        chineseNames[i + 2] = tableTitle;
                    }
                    WriteExcel.writeHead(chineseNames, sheet);
                    JSONArray dataArray = returndataObj.getJSONArray("datanodes");
                    for (int i = 0; i < dataArray.size(); i++) {
                        JSONObject oneObj = dataArray.getJSONObject(i);
                        JSONObject dataObj = oneObj.getJSONObject("data");
                        String data = dataObj.get("data") + "";
                        if ("0".equals(data)) {
                            oneProvanceValueList.add("");
                        } else {
                            int pointIndex = data.indexOf(".");
                            if (pointIndex > 0 && !data.contains("E")) {
                                String integralPart = data.substring(0, pointIndex);
                                String pointPart = data.substring(pointIndex + 1);
                                if (pointPart.length() > 2) {
                                    Double doubleData = Double.parseDouble(data);
                                    BigDecimal bg = new BigDecimal(doubleData).setScale(2, RoundingMode.HALF_EVEN);
                                    oneProvanceValueList.add(bg.doubleValue() + "");
                                } else {
                                    oneProvanceValueList.add(data);
                                }
                            } else if (pointIndex < 0 && !data.contains("E")) {
                                oneProvanceValueList.add(data);
                            }
                            if (data.contains("E")) {
                                BigDecimal bd = new BigDecimal(data);
                                String data_edit = bd.toPlainString();
                                oneProvanceValueList.add(data_edit);
                            }
                        }
                    }
                    WriteExcel.writeExcel(rowNumber, oneProvanceValueList, sheet, momeList);
                } catch (UnsupportedEncodingException e) {
                    e.printStackTrace();
                }
            }
        }
        WriteExcel.exportExcel("E:\\程存淦_处理程序2020\\王佳艺\\CRED\\国家数据\\地区数据_分省年度数据\\", tableName, workbook);
    }

    public static String getContentJsonStr(String contentUrl) {
        try {
            OkHttpClient client = new OkHttpClient();
            Request request = new Request.Builder().url(contentUrl).build();
            Response response = client.newCall(request).execute();
            if (response.isSuccessful()) {
                String jsonStr = response.body().string();
                return jsonStr;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return "";
    }

    private static List<String> getKeySet(Map<String, String> map) {
        Set<String> keySet = map.keySet();//获取表格分类id_名称Map的key值 先转换为set集合
        Iterator<String> iter = keySet.iterator();//迭代器遍历
        List<String> keyList = new ArrayList<String>();
        while (iter.hasNext()) {//判断是够有下一个元素
            String areaId = iter.next();//获取集合元素
            keyList.add(areaId);
        }
        return keyList;
    }

    private static Map<String, String> getMap() {
        String jsonString = AreaJsonString.getJsonString();
        JSONArray jsonArray = JSON.parseArray(jsonString);
        Map<String, String> area_id_map = new HashMap<String, String>();
        for (int i = 0; i < jsonArray.size(); i++) {
            String arrayString = jsonArray.getString(i);//把java json数组转换成json对象，并转换成字符串
            JSONObject jsonObject = JSON.parseObject(arrayString);
            String id = jsonObject.getString("id");
            String name = jsonObject.getString("name");
            area_id_map.put(id, name);//键值对
        }
        return area_id_map;
    }
}
